 ;;************************************************************************
;; datavis.lsp 
;; contains code for deciding on which visualization is appropriate, 
;; and for visualizing univariate or bivariate numeric data  
;; copyright (c) 1999 by Forrest W. Young
;;************************************************************************

;(require (strcat *code-dir-name* "overlay"))

(defun visualize-data (&key (dialog nil dialog?) (menu nil) tour freq univariate bivariate multivariate guided-tour crosstabs classification frequency category relational all-bivariate numeric-bivariate category-bivariate popup-x popup-y)
"Args: Keywords: dialog univariate bivariate multivariate guided-tour tour crosstabs classification frequency freq category relational all-bivariate numeric-bivariate category-bivariate 
Generic Function for VISUALIZE-DATA method. Only one keyword may be used. Action is
(1) shows spreadplot when
    (a) a keyword specifies a specific spreadplot
    (b) only one spreadplot type is possible
(2) Presents choice dialog when
    (a) DIALOG is T and several spreadplots are possible
    (b) No keywords used and several spreadplots are possible"
  (send current-data :visualize-data :dialog dialog :menu menu :tour tour :freq freq :univariate univariate :bivariate bivariate :multivariate multivariate :guided-tour guided-tour :crosstabs crosstabs :classification classification :frequency frequency :category category :relational relational :all-bivariate all-bivariate :numeric-bivariate numeric-bivariate :category-bivariate category-bivariate :popup-x popup-x :popup-y popup-y)
  )

(defun visualize () (send *current-object* :visualize))

(defmeth mv-data-object-proto :visualize (&key dialog tour freq)
"Alias for VISUALIZE-DATA and VISUALIZE-MODEL"
  (send self :visualize-data :dialog dialog :tour tour :freq freq))

(defmeth mv-data-object-proto :visualize-statistical-object (&key dialog)
"Args: (dialog nil)
Generic method used for both data and model objects by toolbar visualize button. Permits data to have dialog and model to not."
  (send current-data :visualize-data :dialog dialog))

 
(defmeth mv-data-object-proto :visualize-data (&key (dialog nil) (menu nil) tour freq univariate bivariate multivariate guided-tour crosstabs classification frequency category relational all-bivariate numeric-bivariate category-bivariate popup-x popup-y) 
"ARGS: &KEY (dialog t) (menu t) tour freq univariate bivariate multivariate guided-tour crosstabs classification frequency category relational all-bivariate numeric-bivariate category-bivariate popup-x popup-y
Constructs specified type of data visualization, if appropriate, or presents dialog or popup menu to determine type of visualization."
  ;(send self :determine-data-type);fwy feb 4,2001
  (when (and (equal (send self :data-type) "missing") 
             (not multivariate)) (visualize-missing-data));PV
  (when (not (and (equal (send self :data-type) "missing") 
             (not multivariate)));PV
        (if (not (eq current-object self)) (setcd self))
        (let* ((icon (select (send *workmap* :icon-list) 
                      (1- (send current-data :icon-number))))
               (sp-list (remove-if #'null (send self :spreadplots)))
               (sp-titles)
               (n-numord (send self :active-nvar '(numeric ordinal)))
               (n-cat (send self :active-nvar '(category)))
               (data-type (send self :data-type))
               (current-data-type (send self :determine-data-type))
               (did-tour) (best-vis-type) (possible-vis-types)
               (resp-var-name)(emulated-table-data)(class-result)
               (class-warning-message) (warning-window)
               (too-many-cats)
               (zero-freqs) (warn-str )
               (choices) (result))
;data types: category class freq multivariate matrix missing
;result of 0=mulvar 1=class 2=cat 3=tour 4=freq 5=crstab    
          (when (and (> n-numord 0) (> n-cat 0))
                (setf class-result (send self :get-class-vis n-numord));check
                (setf emulated-table-data (first class-result))
                (setf resp-var-name (second class-result))
                (setf class-warning-flag (third class-result)))
          (send *workmap* :stop-screen-saver)
          (send icon :do-click "graph")
          (cond
;IF KEYWORD USED, SET INITIAL SPREADPLOT SUGGESTION ACCORDING TO KEYWORD.
;IF KEYWORD NOT USED, SET INITIAL SPREADPLOT SUGGESTION ACCORDING TO DATATYPE. 
            (univariate      (setf result 0))  ;univariate     0
            (bivariate       (setf result 0))  ;bivariate      0
            (multivariate    (setf result 0))  ;multivariate   0
            (classification  (setf result 1))  ;classification 1
            (category        (setf result 2))  ;category       2
            (guided-tour     (setf result 3))  ;guided tour    3
            (tour            (setf result 3))  ;tour           3
            (freq            (setf result 4))  ;freq           4
            (frequency       (setf result 4))  ;freq           4
            (crosstabs       (setf result 5))  ;crosstabs      5 ;2 ;5
            (relational      (setf result 6))  ;relational     6
            (all-bivariate   (setf result 7))  ;all-bivariate  7
            (numeric-bivariate (setf result 8))  ;numeric-bivariate  8
            (category-bivariate (setf result 9))  ;numeric-bivariate  9
            ((and (> n-cat 1)
                  (send self :crosstabs-data?))       (setf result 5));5
            ((EQUAL current-data-type "univariate")   (setf result 0))
            ((EQUAL current-data-type "bivariate")    (setf result 0))
            ((EQUAL current-data-type "multivariate") (setf result 0))   
            ((EQUAL current-data-type "class")        (setf result 1))
            ((EQUAL current-data-type "category")     (setf result 2))
            ((EQUAL current-data-type "general")      (setf result 0))
            ((EQUAL current-data-type "freq")         (setf result 4))
            ((EQUAL current-data-type "crosstabs")    (setf result 5))
            ((EQUAL current-data-type "freqclass")    (setf result 4))
            ((EQUAL current-data-type "matrix")       (setf result 6))
            ((EQUAL current-data-type "new")          (setf result 99))
            )
          (SETF best-vis-type RESULT)

          (setf possible-vis-types (remove 'nil
              (list 
               (when (or (send self :freq) (equal data-type "freq"))
                     (setf possible-vis-types 4))
               (when (and (equal current-data-type "multivariate") (> n-numord 5)) 3)
               (when (and (equal current-data-type "general") 
                          (> n-numord 5)) 3)
               (when (and (> n-numord 0) 
                          (> (send self :nobs) 2)) 0)
               (when (and (> n-numord 0) (> (send self :nobs) 2)
                          (> n-cat 1)) 5)
               (when resp-var-name 1)
               (when (and (not (send self :freq)) (> n-cat 0)) 2)
               )))
            
          (setf possible-vis-types
                (remove 'nil
                 (cond 
                  ((> (length possible-vis-types) 1)
                   (combine best-vis-type 
                            (remove best-vis-type 
                                    Possible-vis-types)))
                  ((= best-vis-type (first possible-vis-types))
                   possible-vis-types)
                  (t 
                   (combine best-vis-type possible-vis-types)))))
             
          (when (member 5 possible-vis-types)
                (setf possible-vis-types (combine 5 (remove 5 possible-vis-types))))

          ;temporarily remove crosstabs
          ;(when (member 5 possible-vis-types)
          ;      (setf possible-vis-types (remove 5 possible-vis-types)))

          (when (member best-vis-type possible-vis-types)
                (setf possible-vis-types (combine best-vis-type
                                                  (remove best-vis-type
                                                   possible-vis-types))))
          (setf possible-vis-types (remove 'nil possible-vis-types))
          (setf possible-vis-types (remove-duplicates possible-vis-types))
          (setf num-types-possible (length possible-vis-types))
          (setf class-result (send self :choose-class-vis n-numord t))
          (setf class-warning-flag (third class-result))
          (if class-warning-flag (setf possible-vis-types (remove 1 possible-vis-types)))

          (send self :possible-vis-types possible-vis-types)

          (when (> (length possible-vis-types) 0)
                (when class-warning-message 
                      (setf warning-window (vista-message class-warning-message)))
                (setf class-warning-message nil)
                (setf choices 
                      (select (list "Numeric Data SpreadPlot"
                                    "Classification Data SpreadPlot"
                                    "Category Data SpreadPlot"
                                    "Guided Tour SpreadPlot"
                                    "Frequency Data SpreadPlot"
                                    "CrossTabulation SpreadPlot"
                                    "Relational Data SpreadPlot")
                              (remove 'nil possible-vis-types))))

          (when (and best-vis-type (> (length choices) 1))
                (setf (select choices 0) 
                      (strcat (select choices 0) " (best choice)")))
          (cond 
            ((= (length choices) 0) 
             (vista-message "Visualization not possible")
             (setf result nil))
            ;((= (length choices) 1) (first possible-vis-types))
            ((and menu dialog) (error "can not use both menu and dialog keywords"))
            (menu
             (let* ((items (send *datavis-popup-menu* :items)))
               (mapcar #'(lambda (item) (send item :enabled nil)) items)
               (mapcar #'(lambda (item) (send item :enabled t))
                       (select items possible-vis-types))
               (setf result (- (send *datavis-popup-menu* 
                                  :popup (floor popup-x) (floor popup-y) *workmap*) 1))))
            (dialog 
             (setf result (select possible-vis-types
                                  (choose-item-dialog 
                                   "Data Visualization Choices:" choices))))
 
            (t (setf result best-vis-type)))
          (when result
                (when (= result 1)
                      (setf class-result (send self :choose-class-vis n-numord nil))
                      (setf class-warning-flag (third class-result))
                     ; (when class-warning-flag (error "impossible branch in datavis.lsp"))
                      (cond
                        (class-warning-flag (setf result 2))
                        (t
                         (setf resp-var-name (second class-result))
                         (setf emulated-table-data (first class-result)))))
                (cond
                  ((= result 99) 
                   (error-message "Visualization is not possible. The data may need to be further cleaned, or there may be a program error."))
                  (t
                   (send self :create-spreadplot-container)
                   (case result
                     (0 (setf did-tour 
                         (send self :visualize-continuous-data dialog nil nil)))
                     (1 (send emulated-table-data :visualize-class-data resp-var-name))
                     (2 (send self :visualize-freq-array))
                     (3 (setf did-tour (send self :visualize-continuous-data dialog t nil)))
                     (4 (send self :visualize-freq-array))
                     (5 (send self :visualize-nway-crosstabs-array))
                     (6 (relational-view)))
                   (disable-container)
                   (send command-menu-refresh-item :enabled nil)
                   (when (and (/= result 99) ;(not did-tour) ;fwy 2-21-2001
                              *current-spreadplot*)
                         (send self :spreadplots 
                               (add-element-to-list (send self :spreadplots)
                                                    *current-spreadplot*)))
                   (send *watcher* :close)
                  ; (send *spreadplot-container* :redraw ())
                   ;(send *spreadplot-container* :max-window)
                   )))
          *current-spreadplot*)))

(defmeth mv-data-object-proto :get-class-vis (n-numord)
  "Returns a list of table-data object, response variable name and error message flag. When third element t, visualization is not possible."
  (let* ((resp-var-name (first (send self :active-variables '(numeric))))
         (emulated-table-data (emulate-table-data resp-var-name self)))
    (list emulated-table-data resp-var-name 
          (send emulated-table-data :incomplete-data-error-flag))))

 
(defmeth mv-data-object-proto :create-spreadplot-container (&optional style)
"Creates a spreadplot container object. Binds *spreadplot-container* to the object. Returns the object."
  (send *watcher* :write-text "Constructing SpreadPlot Container")
  (let ((object (make-container 
                  :size (send *vista* :spreadplot-sizes) 
                  :free *free-spreadplots*
                  :local-menus *free-spreadplots*
                  :type (if style style (if *seamless-spreadplots* 1 7))
                  :show nil))
        )
    (setf *spreadplot-container* object)
    (defmeth object :close () (send self :hide-window))
    object))

(defun create-spreadplot-container (&optional style)
"Creates a spreadplot container object. Binds *spreadplot-container* to the object. Returns the object."
  (send *watcher* :write-text "Constructing SpreadPlot Container")
  (let ((object (make-container 
                  :size (send *vista* :spreadplot-sizes) 
                  :free *free-spreadplots*
                  :local-menus *free-spreadplots*
                  :type (if style style (if *seamless-spreadplots* 1 7))
                  :show nil))
        )
    (setf *spreadplot-container* object)
    (defmeth object :close () (send self :hide-window))
    object))

(defmeth mv-data-object-proto :choose-class-vis (n-numord &optional no-dialog)
  "Presents a dialog listing possible numeric and categorical variables. User chooses one numeric and one or more categorical variables. Choice is checked with check-class-vis for completeness (is the still required?). Returns list of new emulated-table-data object, response varname, or nil to stop."
  (let* ((numvars (send self :active-variables '(numeric)))
         (ncat (length (send self :active-variables '(category))))
         (resp-var-name (first numvars))
         (emulated-table-data (emulate-table-data resp-var-name self))
         (err-flag (send emulated-table-data :incomplete-data-error-flag))
         )
    (cond 
      ((> ncat 4)
       (setf err-flag t)
       (unless no-dialog
               (vista-dialog "Classification visualization cannot use more than four category variables. The first 4 variables will be used."))
       ) 
      ((and err-flag (not no-dialog))
       (HELP (format nil "For the classification visualization you must choose a set of category variables whose cross-tabulation frequencies are all positive - there cannot be any zero frequencies in the cells of the crosstab. ~2%Since there are zero frequencies in the crosstab of these category variables, the classification visualization cannot be shown.~2%This limitation does not apply to the category data visualization, which has been presented instead. ~2%Using the visualization which is shown, you can determine if you can choose fewer category variables whose crosstabs frequencies are all positive. If you can, you can then see the classification visualization for those variables."))
       (send *help-window* :top-most t))
      ((= n-numord 1))
      ((and (> n-numord 1) (not no-dialog))
       (setf resp-var-name 
             (select numvars
                     (choose-item-dialog "Choose a variable to visualize:" numvars)))
       (if resp-var-name
           (setf emulated-table-data (emulate-table-data resp-var-name self))
           (setf err-flag t))))
    (list emulated-table-data resp-var-name err-flag)))
          

(defmeth mv-data-object-proto :check-class-vis (n-numord)
"Checks on whether can do classification visualization. Returns a list of table-data object response variable name and error message. Third element nil when visualization is possible, first two nil when not possible."
  (let* ((errmsg1) (errmsg2) (errmsg) (resp-var-name) (emulated-table-data)
         (info (format nil "~2%Classification data are data where values on 1 numeric variable are classified by one or more category variables.")))
   ; (if (> n-numord 1)
    ;    (setf errmsg1 "when there is exactly 1 active numeric variable"))
    (setf resp-var-name 
          (first (send self :active-variables '(numeric))))
    (setf emulated-table-data 
          (emulate-table-data resp-var-name self))

    (if (send emulated-table-data :incomplete-data-error-flag)
        (setf errmsg2 (format nil "when the data are complete.~2%Data are \"complete\" when the frequency table calculated from the active category variables has no cells with zero frequencies)")))
    (cond 
      ((or errmsg1 errmsg2)
       (setf errmsg (strcat "A visualization for classification data can be created "
                            errmsg1 (if (and errmsg1 errmsg2) " and " "")
                            errmsg2 (format nil ".~2%To see the classification visualization you must choose fewer ~a" (if errmsg1 "numeric" ""))
                            (if (and errmsg1 errmsg2) " and " "")
                            (if errmsg2 "category" "") " variables."
                            info))
       
       (list nil nil errmsg))
      (t
       (list emulated-table-data resp-var-name nil)))))


  
(defmeth mv-data-object-proto :visualize-continuous-data (dialog tour freq)
  (let ((n-numord (send self :active-nvar '(numeric ordinal)))
        (n-cat (send self :active-nvar '(category)))
        (choice 0)
        (did-tour)
        (obsvar-states nil))
    (cond 
;univariate data
      ((= n-numord 1)
       (send self :visualize-one-variable
             (send self :active-data '(numeric ordinal))
             (first (send self :active-variables '(numeric ordinal)))
             (send self :active-labels)))
;bivariate data
      ((= n-numord 2)
       (send self :visualize-two-variables
             (column-list (send self :active-data-matrix '(numeric ordinal)))
             (send self :active-variables '(numeric ordinal))
             (send self :active-labels)))
;multivariate data      
      ((< n-numord 6)
       (send self :visualize-n-variables 
             (send self :active-data-matrix '(numeric ordinal))
             (send self :active-variables '(ordinal numeric))
             (send self :active-labels)))
;tour plot
      (tour
       (send self :visualize-tour-plot)
       (setf did-tour t)
       (setf tour t))
      (t 
       (send self :visualize-n-variables 
             (send self :active-data-matrix '(numeric ordinal))
             (send self :active-variables '(ordinal numeric))
             (send self :active-labels))))
    did-tour))

(defmeth mv-data-object-proto :vis-error-msg (word)
  (vista-message 
(format nil "The category variable(s) were not used. To use them, you can either:~2%Select just one numeric or ordinal variable and one or more category variables. Then you can ~a the categories of the numeric or ordinal variable;~2%Select only category variables. Then you can ~a the frequencies of the category cross-classifications." word word)
   ) t)

(defmeth mv-data-object-proto :vis-error-msg2 ()
  (vista-message 
(format nil "The cross-classification of the category variables is not complete: It has cells with zero frequency.~2%Currently, you cannot use these category variables in conjunction with a numeric or ordinal variable.~2%However, you can use them by themselves to visualize the frequencies of the category cross-classifications. To do this, select only the category variables.")
   ) t)

(defmeth mv-data-object-proto :visualize-two-variables 
  (vars variable-labels point-labels)
  (let* ((varx (select vars 0))
         (vary (select vars 1))
         (sp (spread-plot 
              (matrix '(2 3) 
                      (list
                       (plot-points
                        varx vary 
                        :point-labels point-labels
                        :variable-labels variable-labels
                        :title "Scatterplot" :menu-title "Scatter"
                        :show nil)
                       (quantile-plot 
                        (list varx vary) 
                        :reg-line t 
                        :variable-labels variable-labels
                        :point-labels point-labels 
                        :show nil)
                       (quantile-quantile-plot 
                        varx vary
                        :reg-line t
                        :variable-labels variable-labels
                        :point-labels point-labels
                        :show nil)
                       (boxplot 
                        (list varx vary) 
                        :variable-labels variable-labels
                        :point-labels point-labels
                        :equate t
                        :connect-points t
                        :show nil)
                       (histofreq 
                        (list varx vary) :new-x t
                        :variable-labels variable-labels
                        :show nil)
                       (name-list point-labels :show nil)
                       ))))
         (plot-matrix (send sp :plot-matrix))
         (scatterplot (select plot-matrix 0 0))
         (q-plot (select plot-matrix 0 1))
         (qq-plot (select plot-matrix 0 2))
         (box-plot (select plot-matrix 1 0))
         (histofreq (select plot-matrix 1 1))
         (obs-list (select plot-matrix 1 2))
         (nobs (send self :nobs))
         )
    ;(send sp :title (strcat "Bivariate SpreadPlot: " (send *current-spreadplot* :title)))
    (defmeth sp :spreadplot-help ()
      (plot-help-window (strcat "SpreadPlot Help"))
      (paste-plot-help (format nil "This is the SpreadPlot for Bivariate Data. In this SpreadPlot the windows are only linked by the data's observations.~2%"))
      (call-next-method :points t :bars t :labels t :flush nil))

    (send box-plot :linked t)
    (send box-plot :showing-labels t)
    (send box-plot :use-color t)
    (send box-plot :point-color (iseq (send box-plot :num-points)) 'blue)
    (send box-plot :mouse-mode 'brushing)
    (send box-plot :connect-points t)

    (send obs-list :linked t)
    (send obs-list :title "Observations")
    (send obs-list :new-menu "Obs" 
          :items '(LINK MOUSE DASH ERASE-SELECTION FOCUS-ON-SELECTION 
                        SHOW-ALL COLOR SELECTION))
    (send obs-list :fix-name-list)
    (send obs-list :use-color t)
    (send obs-list :point-color (iseq nobs) 'blue)

    (send scatterplot :scale-type 'variable)
    (send scatterplot :vista-look-and-feel)
    (send q-plot    :plot-buttons :new-x nil :new-y nil :mouse-mode t)
    (send q-plot    :linked t)
    (send qq-plot   :plot-buttons :new-x nil :new-y nil :mouse-mode nil)
    (send box-plot  :redraw)
    (send q-plot    :redraw)
    (send qq-plot   :redraw)
    (send obs-list  :redraw)
    (send histofreq :redraw)
    (send scatterplot :redraw)
    (send sp :show-spreadplot)
    ))

(defmeth mv-data-object-proto :visualize-one-variable 
  (var variable-labels point-labels)
  (let* ((sp (spread-plot 
              (matrix '(2 2) 
                      (list
                       (boxplot var 
                                :variable-labels (list variable-labels)
                                :point-labels point-labels
                                :show nil)
                       (name-list point-labels 
                                  :show nil :title "Observations")
                       (quantile-plot var 
                                :reg-line t 
                                :variable-label variable-labels
                                :point-labels point-labels 
                                :show nil)
                       (histofreq var 
                                  :variable-labels variable-labels
                                  :show nil)
                       ))))
         (plot-matrix (send sp :plot-matrix))
         (box-plot (select plot-matrix 0 0))
         (obs-list (select plot-matrix 0 1))
         (q-plot (select plot-matrix 1 0))
         (freq-plot (select plot-matrix 1 1))
         (nobs (send self :nobs))
         )
   ; (send sp :title (strcat "Univariate SpreadPlot: " (send *current-spreadplot* :title)))
    (defmeth sp :spreadplot-help ()
      (plot-help-window (strcat "SpreadPlot Help"))
      (paste-plot-help (format nil "This is the SpreadPlot for Univariate Data. In this SpreadPlot the windows are only linked by the data's observations.~2%"))
      (call-next-method :points t :bars t :labels t :flush nil))
    (send box-plot :linked t)
    (send box-plot :showing-labels t)
    (send box-plot :use-color t)
    (send box-plot :point-color (iseq (send box-plot :num-points)) 'blue)
    (send box-plot :mouse-mode 'brushing)
    
    (send obs-list :linked t)
    (send obs-list :new-menu "Obs" 
          :items '(LINK MOUSE DASH ERASE-SELECTION FOCUS-ON-SELECTION 
                        SHOW-ALL COLOR SELECTION))
    (send obs-list :fix-name-list)
    (send obs-list :use-color t)
    (send obs-list :point-color (iseq nobs) 'blue)
    (send q-plot   :margin 0 17 0 0)
    (send q-plot   :plot-buttons :new-x nil :new-y nil :mouse-mode t)
    (send q-plot    :linked t)
    (send freq-plot :use-color t)
    (send box-plot :redraw)
    (send q-plot :redraw)
    (send freq-plot :redraw)
    (send obs-list :redraw)
    (send sp :show-spreadplot)
    ))